from kateglo_crawler.spiders import arrToStr
from kateglo_crawler.spiders import sanitize
from scrapy.http.request import Request
from scrapy.selector.libxml2sel import HtmlXPathSelector
from scrapy.spider import BaseSpider

from __init__ import *
from kateglo_crawler.items import DictionaryItem
from kateglo_crawler.spiders import get_dictionary_item


#
# DICTIONARY DETAIL CRAWL
#
class DictionaryDetailCrawler(BaseSpider):
    # Identifier
    name = 'dictionary_detail'

    # Debug
    debug = False

    # Rules
    allowed_domains = ['kateglo.bahtera.org']
    start_urls = []

    def start_requests(self):
        new_dictionary_item = get_dictionary_item()
        url = 'http://kateglo.bahtera.org/?mod=dictionary&action=view&phrase=' + new_dictionary_item[1]
        request = Request(url, dont_filter=True)
        yield request

    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        dictionary_item = DictionaryItem()
        dictionary_item['phrase'] = sanitize(arrToStr(hxs.select('//h1//text()').extract()))
        dictionary_item['definition'] = sanitize(arrToStr(hxs.select('//p[starts-with(.//text(),"Definisi:")]//following-sibling::*[1]//text()').extract()))
        dictionary_item['class_phrase'] = sanitize(arrToStr(hxs.select('//p[starts-with(.//text(),"Kelas:")]//following-sibling::*[1]//text()').extract()))
        dictionary_item['root'] = sanitize(arrToStr(hxs.select('//p[starts-with(.//text(),"Kata dasar:")]//following-sibling::*[1]//text()').extract()))
        dictionary_item['source'] = sanitize(arrToStr(hxs.select('//p[starts-with(.//text(),"Sumber:")]//following-sibling::*[1]//text()').extract()))
        dictionary_item['urldetail'] = response.url
        dictionary_item['is_detailed'] = 1
        yield dictionary_item

        new_dictionary_item = get_dictionary_item()
        url = 'http://kateglo.bahtera.org/?mod=dictionary&action=view&phrase=' + new_dictionary_item[1]
        request = Request(url, dont_filter=True)
        yield request